{
 "cells": [
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": [
    "%env LLM_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1\n",
    "%env LLM_API_KEY=替换为自己的Qwen API Key\n",
    "%env ZHIPU_API_KEY=替换为自己的智谱API Key，如果不需要测试智谱可以不输入并跳过相应代码"
   ],
   "id": "3bd0a3f61ef3e0e3"
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "af375836-b870-458b-87d1-4e00565977eb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-11T15:30:33.659922Z",
     "iopub.status.busy": "2024-10-11T15:30:33.659650Z",
     "iopub.status.idle": "2024-10-11T15:30:33.665559Z",
     "shell.execute_reply": "2024-10-11T15:30:33.665093Z",
     "shell.execute_reply.started": "2024-10-11T15:30:33.659907Z"
    }
   },
   "outputs": [],
   "source": [
    "%%capture --no-stderr\n",
    "!pip install -U langchain langchain_community langchain_openai pypdf sentence_transformers chromadb shutil openpyxl FlagEmbedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "1e2c72b8-ee12-4130-af88-699998aa230c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-11T15:30:33.666701Z",
     "iopub.status.busy": "2024-10-11T15:30:33.666562Z",
     "iopub.status.idle": "2024-10-11T15:30:33.873840Z",
     "shell.execute_reply": "2024-10-11T15:30:33.873382Z",
     "shell.execute_reply.started": "2024-10-11T15:30:33.666688Z"
    }
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "841d2b02-ad06-40d2-b11f-c7adccec6ca2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-11T15:30:33.874454Z",
     "iopub.status.busy": "2024-10-11T15:30:33.874289Z",
     "iopub.status.idle": "2024-10-11T15:30:33.877112Z",
     "shell.execute_reply": "2024-10-11T15:30:33.876755Z",
     "shell.execute_reply.started": "2024-10-11T15:30:33.874441Z"
    }
   },
   "outputs": [],
   "source": [
    "expr_version = 'generation_v1_long_context_no_rag'\n",
    "\n",
    "preprocess_output_dir = os.path.join(os.path.pardir, 'outputs', 'v1_20240713')\n",
    "expr_dir = os.path.join(os.path.pardir, 'experiments', expr_version)\n",
    "\n",
    "os.makedirs(expr_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cf7e81e3-4c82-4842-aef5-7592caaf1d39",
   "metadata": {},
   "source": [
    "# 读取文档"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e6920e29-bc7d-4635-be06-d151eaf0e100",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-11T15:30:33.877643Z",
     "iopub.status.busy": "2024-10-11T15:30:33.877525Z",
     "iopub.status.idle": "2024-10-11T15:30:35.402783Z",
     "shell.execute_reply": "2024-10-11T15:30:35.402340Z",
     "shell.execute_reply.started": "2024-10-11T15:30:33.877631Z"
    }
   },
   "outputs": [],
   "source": [
    "from langchain_community.document_loaders import PyPDFLoader\n",
    "from langchain.schema import Document\n",
    "\n",
    "loader = PyPDFLoader(os.path.join(os.path.pardir, 'data', '2024全球经济金融展望报告.pdf'))\n",
    "documents = loader.load()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "5cfd29e8-808b-4c1d-b41a-7e6926e3d5be",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-11T15:30:35.403688Z",
     "iopub.status.busy": "2024-10-11T15:30:35.403343Z",
     "iopub.status.idle": "2024-10-11T15:30:35.405798Z",
     "shell.execute_reply": "2024-10-11T15:30:35.405498Z",
     "shell.execute_reply.started": "2024-10-11T15:30:35.403675Z"
    }
   },
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "pattern = r\"^全球经济金融展望报告\\n中国银行研究院 \\d+ 2024年\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "8fc67220-a59d-48d9-a689-3205189ad693",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-11T15:30:35.407019Z",
     "iopub.status.busy": "2024-10-11T15:30:35.406855Z",
     "iopub.status.idle": "2024-10-11T15:30:35.422525Z",
     "shell.execute_reply": "2024-10-11T15:30:35.421355Z",
     "shell.execute_reply.started": "2024-10-11T15:30:35.407007Z"
    }
   },
   "outputs": [],
   "source": [
    "docs = [Document(page_content='\\n'.join(re.sub(pattern, '', doc.page_content) for doc in documents))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "f5c7735f-4271-4f18-b932-f45e5283f016",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-11T15:30:35.424038Z",
     "iopub.status.busy": "2024-10-11T15:30:35.423679Z",
     "iopub.status.idle": "2024-10-11T15:30:35.538821Z",
     "shell.execute_reply": "2024-10-11T15:30:35.538386Z",
     "shell.execute_reply.started": "2024-10-11T15:30:35.424003Z"
    }
   },
   "outputs": [],
   "source": [
    "qa_df = pd.read_excel(os.path.join(preprocess_output_dir, 'question_answer.xlsx'))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7925564a-7d30-4914-baaf-4a00abb7686d",
   "metadata": {},
   "source": [
    "# 生成答案"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "f26abaf0-b0f8-46b4-9b69-a5e28cf1b7b1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-11T15:30:35.539473Z",
     "iopub.status.busy": "2024-10-11T15:30:35.539311Z",
     "iopub.status.idle": "2024-10-11T15:30:35.585703Z",
     "shell.execute_reply": "2024-10-11T15:30:35.585238Z",
     "shell.execute_reply.started": "2024-10-11T15:30:35.539460Z"
    }
   },
   "outputs": [],
   "source": [
    "from langchain.llms import Ollama\n",
    "\n",
    "qwen2_7b_llm = Ollama(\n",
    "    model='qwen2:7b-instruct',\n",
    "    base_url=\"http://localhost:11434\"\n",
    ")\n",
    "qwen2_7b_32k_llm = Ollama(\n",
    "    model='qwen2:7b-instruct-32k',\n",
    "    base_url=\"http://localhost:11434\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "69846ccf-5fca-47a5-91c1-7099c6734ce1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-11T15:30:35.586355Z",
     "iopub.status.busy": "2024-10-11T15:30:35.586162Z",
     "iopub.status.idle": "2024-10-11T15:30:35.923613Z",
     "shell.execute_reply": "2024-10-11T15:30:35.923094Z",
     "shell.execute_reply.started": "2024-10-11T15:30:35.586342Z"
    }
   },
   "outputs": [],
   "source": [
    "from langchain_openai import ChatOpenAI\n",
    "\n",
    "glm4_plus_llm = ChatOpenAI(\n",
    "    model='glm-4-plus',\n",
    "    base_url='https://open.bigmodel.cn/api/paas/v4/',\n",
    "    api_key=os.environ['ZHIPU_API_KEY']\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "2d044d99-b3dc-4e3b-9172-65084a755824",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-11T15:30:35.924306Z",
     "iopub.status.busy": "2024-10-11T15:30:35.924175Z",
     "iopub.status.idle": "2024-10-11T15:30:35.926831Z",
     "shell.execute_reply": "2024-10-11T15:30:35.926526Z",
     "shell.execute_reply.started": "2024-10-11T15:30:35.924293Z"
    }
   },
   "outputs": [],
   "source": [
    "prompt_tmpl = \"\"\"\n",
    "你是一个金融分析师，擅长根据所获取的信息片段，对问题进行分析和推理。\n",
    "你的任务是根据所获取的信息片段（<<<<context>>><<<</context>>>之间的内容）回答问题。\n",
    "回答保持简洁，不必重复问题，不要添加描述性解释和与答案无关的任何内容。\n",
    "已知信息：\n",
    "<<<<context>>>\n",
    "{{knowledge}}\n",
    "<<<</context>>>\n",
    "问题：{{query}}\n",
    "请回答：\n",
    "\"\"\".replace('{{knowledge}}', '\\n\\n'.join([doc.page_content for doc in docs])).strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "246280fb-053f-43ea-8f47-126a76cccc20",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-11T15:30:36.572115Z",
     "iopub.status.busy": "2024-10-11T15:30:36.571333Z",
     "iopub.status.idle": "2024-10-11T15:30:36.578595Z",
     "shell.execute_reply": "2024-10-11T15:30:36.578256Z",
     "shell.execute_reply.started": "2024-10-11T15:30:36.572045Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "31333"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(prompt_tmpl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "1066787e-f803-4501-a002-33e71ab64a75",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-11T13:28:52.530967Z",
     "iopub.status.busy": "2024-10-11T13:28:52.530841Z",
     "iopub.status.idle": "2024-10-11T13:28:52.544028Z",
     "shell.execute_reply": "2024-10-11T13:28:52.543611Z",
     "shell.execute_reply.started": "2024-10-11T13:28:52.530955Z"
    }
   },
   "outputs": [],
   "source": [
    "def get_answer(llm, query):\n",
    "    prompt = prompt_tmpl.replace('{{query}}', query)\n",
    "\n",
    "    resp = llm.invoke(prompt)\n",
    "    return resp if isinstance(resp, str) else resp.content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "7ce53a5a-25a3-4011-bec2-97b00f7afc93",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-11T13:28:52.544582Z",
     "iopub.status.busy": "2024-10-11T13:28:52.544461Z",
     "iopub.status.idle": "2024-10-11T13:28:55.539740Z",
     "shell.execute_reply": "2024-10-11T13:28:55.539272Z",
     "shell.execute_reply.started": "2024-10-11T13:28:52.544570Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'报告的发布机构是中国银行研究院。'"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_answer(qwen2_7b_llm, '报告的发布机构是什么？')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "f8cf45bf-3abc-4e38-a798-b95af0db328f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-11T13:28:55.541480Z",
     "iopub.status.busy": "2024-10-11T13:28:55.541348Z",
     "iopub.status.idle": "2024-10-11T13:29:37.930851Z",
     "shell.execute_reply": "2024-10-11T13:29:37.930487Z",
     "shell.execute_reply.started": "2024-10-11T13:28:55.541467Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'报告的发布机构是中国银行研究院。'"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_answer(qwen2_7b_32k_llm, '报告的发布机构是什么？')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "f97bec63-1e9d-4628-a82a-d349ed388c65",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-11T13:29:37.931504Z",
     "iopub.status.busy": "2024-10-11T13:29:37.931376Z",
     "iopub.status.idle": "2024-10-11T13:29:44.664067Z",
     "shell.execute_reply": "2024-10-11T13:29:44.663269Z",
     "shell.execute_reply.started": "2024-10-11T13:29:37.931491Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'中国银行研究院'"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_answer(glm4_plus_llm, '报告的发布机构是什么？')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "95e5a804-2dc6-411c-ba71-6ccf765b2b73",
   "metadata": {},
   "source": [
    "## 预测"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "166392d8-f801-4372-b8ad-3e79aef0b350",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-11T13:29:44.665482Z",
     "iopub.status.busy": "2024-10-11T13:29:44.664954Z",
     "iopub.status.idle": "2024-10-11T13:29:44.722604Z",
     "shell.execute_reply": "2024-10-11T13:29:44.722203Z",
     "shell.execute_reply.started": "2024-10-11T13:29:44.665457Z"
    }
   },
   "outputs": [],
   "source": [
    "from tqdm.auto import tqdm\n",
    "\n",
    "prediction_df = qa_df[qa_df['dataset'] == 'test'][['uuid', 'question', 'qa_type', 'answer']].rename(columns={'answer': 'ref_answer'})\n",
    "\n",
    "def predict(llm, prediction_df):\n",
    "    prediction_df = prediction_df.copy()\n",
    "    answer_dict = {}\n",
    "\n",
    "    for idx, row in tqdm(prediction_df.iterrows(), total=len(prediction_df)):\n",
    "        uuid = row['uuid']\n",
    "        question = row['question']\n",
    "        answer = get_answer(llm, question)\n",
    "        answer_dict[question] = {\n",
    "            'uuid': uuid,\n",
    "            'ref_answer': row['ref_answer'],\n",
    "            'gen_answer': answer\n",
    "        }\n",
    "        \n",
    "    prediction_df.loc[:, 'gen_answer'] = prediction_df['question'].apply(lambda q: answer_dict[q]['gen_answer'])\n",
    "    return prediction_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "2fea7d8e-6885-403d-a009-c894b0f8ba21",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-11T13:29:44.723215Z",
     "iopub.status.busy": "2024-10-11T13:29:44.723089Z",
     "iopub.status.idle": "2024-10-11T13:29:44.725266Z",
     "shell.execute_reply": "2024-10-11T13:29:44.724961Z",
     "shell.execute_reply.started": "2024-10-11T13:29:44.723202Z"
    }
   },
   "outputs": [],
   "source": [
    "pred_dfs = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "8c1f0773-f62b-48c7-b499-eb810bbff4c1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-11T13:29:44.725836Z",
     "iopub.status.busy": "2024-10-11T13:29:44.725682Z",
     "iopub.status.idle": "2024-10-11T13:37:56.233866Z",
     "shell.execute_reply": "2024-10-11T13:37:56.233419Z",
     "shell.execute_reply.started": "2024-10-11T13:29:44.725825Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7b36b0d89f084f68b11a1474689eba88",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pred_dfs['ollama_qwen2_7b'] = predict(qwen2_7b_llm, prediction_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "5011f19e-d7ab-4a00-8ec2-a301288f8566",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-11T13:38:40.769583Z",
     "iopub.status.busy": "2024-10-11T13:38:40.768827Z",
     "iopub.status.idle": "2024-10-11T13:54:18.264856Z",
     "shell.execute_reply": "2024-10-11T13:54:18.264391Z",
     "shell.execute_reply.started": "2024-10-11T13:38:40.769514Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7d4d0ae397ab4d66977081f2e7e1af81",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pred_dfs['ollama_qwen2_7b_32k'] = predict(qwen2_7b_32k_llm, prediction_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2b0283d8-7fec-4ffe-88a0-4d7e83adeee2",
   "metadata": {},
   "source": [
    "**特别注意：**glm4 plus比较贵，百万token需要50元，使用长上下文批量预测，如果不是赠送的token就不要尝试了，一定要观察自己的剩余token，余额为负了不会实时提醒，到时候可能会欠很多钱"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "c742cb85-9f90-4397-a64c-b9deff23bcac",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-11T13:54:27.935246Z",
     "iopub.status.busy": "2024-10-11T13:54:27.934449Z",
     "iopub.status.idle": "2024-10-11T14:06:23.510052Z",
     "shell.execute_reply": "2024-10-11T14:06:23.509705Z",
     "shell.execute_reply.started": "2024-10-11T13:54:27.935174Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f82fc6dc79e84d429ea894bcfba9fd33",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pred_dfs['glm4_plus'] = predict(glm4_plus_llm, prediction_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "5e8a4418-0308-4e1e-8075-b767d6e2f79b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-11T14:06:23.510859Z",
     "iopub.status.busy": "2024-10-11T14:06:23.510725Z",
     "iopub.status.idle": "2024-10-11T14:06:23.517048Z",
     "shell.execute_reply": "2024-10-11T14:06:23.516720Z",
     "shell.execute_reply.started": "2024-10-11T14:06:23.510846Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uuid</th>\n",
       "      <th>question</th>\n",
       "      <th>qa_type</th>\n",
       "      <th>ref_answer</th>\n",
       "      <th>gen_answer</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>e73a0c9d-d42b-4350-a4c3-b38bf67c68a5</td>\n",
       "      <td>报告的发布机构是什么？</td>\n",
       "      <td>detailed</td>\n",
       "      <td>中国银行研究院</td>\n",
       "      <td>报告的发布机构是中国银行研究院。</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>e73a0c9d-d42b-4350-a4c3-b38bf67c68a5</td>\n",
       "      <td>报告的发布日期是什么时候？</td>\n",
       "      <td>detailed</td>\n",
       "      <td>2023年12月12日</td>\n",
       "      <td>报告中未直接提供具体的发布日期。通常，这样的信息会在报告的封面上或前言部分明确标注。由于没有...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>e73a0c9d-d42b-4350-a4c3-b38bf67c68a5</td>\n",
       "      <td>2023年全球经济增长有什么特点？</td>\n",
       "      <td>detailed</td>\n",
       "      <td>全球经济增长动力持续回落，各国复苏分化，发达经济体增速放缓，新兴经济体表现稳定。</td>\n",
       "      <td>由于提供的文本内容与2023年全球经济增长的特点无关，我将提供一个通用的经济趋势分析。\\n\\...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>e73a0c9d-d42b-4350-a4c3-b38bf67c68a5</td>\n",
       "      <td>全球贸易增长情况如何？</td>\n",
       "      <td>detailed</td>\n",
       "      <td>全球贸易增长乏力。</td>\n",
       "      <td>关于全球贸易增长的具体数据和趋势分析需要查阅最新的经济报告或者统计数据来获取最准确的信息。通...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>e73a0c9d-d42b-4350-a4c3-b38bf67c68a5</td>\n",
       "      <td>展望2024年，全球经济复苏的预期如何？</td>\n",
       "      <td>detailed</td>\n",
       "      <td>全球经济复苏预计将依旧疲软。</td>\n",
       "      <td>文章中未直接提供对2024年全球经济复苏的具体预测或预期。文中主要分析了美国房地产市场面临的...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   uuid              question   qa_type  \\\n",
       "0  e73a0c9d-d42b-4350-a4c3-b38bf67c68a5           报告的发布机构是什么？  detailed   \n",
       "1  e73a0c9d-d42b-4350-a4c3-b38bf67c68a5         报告的发布日期是什么时候？  detailed   \n",
       "2  e73a0c9d-d42b-4350-a4c3-b38bf67c68a5     2023年全球经济增长有什么特点？  detailed   \n",
       "3  e73a0c9d-d42b-4350-a4c3-b38bf67c68a5           全球贸易增长情况如何？  detailed   \n",
       "5  e73a0c9d-d42b-4350-a4c3-b38bf67c68a5  展望2024年，全球经济复苏的预期如何？  detailed   \n",
       "\n",
       "                                 ref_answer  \\\n",
       "0                                   中国银行研究院   \n",
       "1                               2023年12月12日   \n",
       "2  全球经济增长动力持续回落，各国复苏分化，发达经济体增速放缓，新兴经济体表现稳定。   \n",
       "3                                 全球贸易增长乏力。   \n",
       "5                            全球经济复苏预计将依旧疲软。   \n",
       "\n",
       "                                          gen_answer  \n",
       "0                                   报告的发布机构是中国银行研究院。  \n",
       "1  报告中未直接提供具体的发布日期。通常，这样的信息会在报告的封面上或前言部分明确标注。由于没有...  \n",
       "2  由于提供的文本内容与2023年全球经济增长的特点无关，我将提供一个通用的经济趋势分析。\\n\\...  \n",
       "3  关于全球贸易增长的具体数据和趋势分析需要查阅最新的经济报告或者统计数据来获取最准确的信息。通...  \n",
       "5  文章中未直接提供对2024年全球经济复苏的具体预测或预期。文中主要分析了美国房地产市场面临的...  "
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_dfs['ollama_qwen2_7b'].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "9116b365-d8bc-4224-94c3-5ac5a2e17b42",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-11T14:06:38.357644Z",
     "iopub.status.busy": "2024-10-11T14:06:38.357424Z",
     "iopub.status.idle": "2024-10-11T14:06:38.364446Z",
     "shell.execute_reply": "2024-10-11T14:06:38.364002Z",
     "shell.execute_reply.started": "2024-10-11T14:06:38.357625Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uuid</th>\n",
       "      <th>question</th>\n",
       "      <th>qa_type</th>\n",
       "      <th>ref_answer</th>\n",
       "      <th>gen_answer</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>e73a0c9d-d42b-4350-a4c3-b38bf67c68a5</td>\n",
       "      <td>报告的发布机构是什么？</td>\n",
       "      <td>detailed</td>\n",
       "      <td>中国银行研究院</td>\n",
       "      <td>报告的发布机构是中国银行研究院。</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>e73a0c9d-d42b-4350-a4c3-b38bf67c68a5</td>\n",
       "      <td>报告的发布日期是什么时候？</td>\n",
       "      <td>detailed</td>\n",
       "      <td>2023年12月12日</td>\n",
       "      <td>报告的发布日期是2023年12月12日。</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>e73a0c9d-d42b-4350-a4c3-b38bf67c68a5</td>\n",
       "      <td>2023年全球经济增长有什么特点？</td>\n",
       "      <td>detailed</td>\n",
       "      <td>全球经济增长动力持续回落，各国复苏分化，发达经济体增速放缓，新兴经济体表现稳定。</td>\n",
       "      <td>根据报告，2023年全球经济增长动力持续回落，各国复苏分化：\\n\\n1. 各国经济增长存在较...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>e73a0c9d-d42b-4350-a4c3-b38bf67c68a5</td>\n",
       "      <td>全球贸易增长情况如何？</td>\n",
       "      <td>detailed</td>\n",
       "      <td>全球贸易增长乏力。</td>\n",
       "      <td>根据文中信息，全球贸易增长乏力。具体来说：\\n\\n在全球供应链持续恢复的同时，生产景气度逐渐...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>e73a0c9d-d42b-4350-a4c3-b38bf67c68a5</td>\n",
       "      <td>展望2024年，全球经济复苏的预期如何？</td>\n",
       "      <td>detailed</td>\n",
       "      <td>全球经济复苏预计将依旧疲软。</td>\n",
       "      <td>根据文中内容，对于2024年全球经济增长的主要预期是：\\n\\n“复苏+分化”将成为2024年...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   uuid              question   qa_type  \\\n",
       "0  e73a0c9d-d42b-4350-a4c3-b38bf67c68a5           报告的发布机构是什么？  detailed   \n",
       "1  e73a0c9d-d42b-4350-a4c3-b38bf67c68a5         报告的发布日期是什么时候？  detailed   \n",
       "2  e73a0c9d-d42b-4350-a4c3-b38bf67c68a5     2023年全球经济增长有什么特点？  detailed   \n",
       "3  e73a0c9d-d42b-4350-a4c3-b38bf67c68a5           全球贸易增长情况如何？  detailed   \n",
       "5  e73a0c9d-d42b-4350-a4c3-b38bf67c68a5  展望2024年，全球经济复苏的预期如何？  detailed   \n",
       "\n",
       "                                 ref_answer  \\\n",
       "0                                   中国银行研究院   \n",
       "1                               2023年12月12日   \n",
       "2  全球经济增长动力持续回落，各国复苏分化，发达经济体增速放缓，新兴经济体表现稳定。   \n",
       "3                                 全球贸易增长乏力。   \n",
       "5                            全球经济复苏预计将依旧疲软。   \n",
       "\n",
       "                                          gen_answer  \n",
       "0                                   报告的发布机构是中国银行研究院。  \n",
       "1                               报告的发布日期是2023年12月12日。  \n",
       "2  根据报告，2023年全球经济增长动力持续回落，各国复苏分化：\\n\\n1. 各国经济增长存在较...  \n",
       "3  根据文中信息，全球贸易增长乏力。具体来说：\\n\\n在全球供应链持续恢复的同时，生产景气度逐渐...  \n",
       "5  根据文中内容，对于2024年全球经济增长的主要预期是：\\n\\n“复苏+分化”将成为2024年...  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_dfs['ollama_qwen2_7b_32k'].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "b1e16119-c193-4290-9dff-4c283f1623d0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-11T14:06:42.014762Z",
     "iopub.status.busy": "2024-10-11T14:06:42.013462Z",
     "iopub.status.idle": "2024-10-11T14:06:42.041934Z",
     "shell.execute_reply": "2024-10-11T14:06:42.039415Z",
     "shell.execute_reply.started": "2024-10-11T14:06:42.014685Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uuid</th>\n",
       "      <th>question</th>\n",
       "      <th>qa_type</th>\n",
       "      <th>ref_answer</th>\n",
       "      <th>gen_answer</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>e73a0c9d-d42b-4350-a4c3-b38bf67c68a5</td>\n",
       "      <td>报告的发布机构是什么？</td>\n",
       "      <td>detailed</td>\n",
       "      <td>中国银行研究院</td>\n",
       "      <td>中国银行研究院</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>e73a0c9d-d42b-4350-a4c3-b38bf67c68a5</td>\n",
       "      <td>报告的发布日期是什么时候？</td>\n",
       "      <td>detailed</td>\n",
       "      <td>2023年12月12日</td>\n",
       "      <td>2023年12月12日</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>e73a0c9d-d42b-4350-a4c3-b38bf67c68a5</td>\n",
       "      <td>2023年全球经济增长有什么特点？</td>\n",
       "      <td>detailed</td>\n",
       "      <td>全球经济增长动力持续回落，各国复苏分化，发达经济体增速放缓，新兴经济体表现稳定。</td>\n",
       "      <td>2023年全球经济增长动力持续回落，经济增速连续两年下降。</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>e73a0c9d-d42b-4350-a4c3-b38bf67c68a5</td>\n",
       "      <td>全球贸易增长情况如何？</td>\n",
       "      <td>detailed</td>\n",
       "      <td>全球贸易增长乏力。</td>\n",
       "      <td>全球贸易增长乏力，主要经济体出口贸易同比增速下降。</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>e73a0c9d-d42b-4350-a4c3-b38bf67c68a5</td>\n",
       "      <td>展望2024年，全球经济复苏的预期如何？</td>\n",
       "      <td>detailed</td>\n",
       "      <td>全球经济复苏预计将依旧疲软。</td>\n",
       "      <td>预计2024年全球经济复苏将依旧疲软，主要经济体增长态势和货币政策走势将进一步分化。</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   uuid              question   qa_type  \\\n",
       "0  e73a0c9d-d42b-4350-a4c3-b38bf67c68a5           报告的发布机构是什么？  detailed   \n",
       "1  e73a0c9d-d42b-4350-a4c3-b38bf67c68a5         报告的发布日期是什么时候？  detailed   \n",
       "2  e73a0c9d-d42b-4350-a4c3-b38bf67c68a5     2023年全球经济增长有什么特点？  detailed   \n",
       "3  e73a0c9d-d42b-4350-a4c3-b38bf67c68a5           全球贸易增长情况如何？  detailed   \n",
       "5  e73a0c9d-d42b-4350-a4c3-b38bf67c68a5  展望2024年，全球经济复苏的预期如何？  detailed   \n",
       "\n",
       "                                 ref_answer  \\\n",
       "0                                   中国银行研究院   \n",
       "1                               2023年12月12日   \n",
       "2  全球经济增长动力持续回落，各国复苏分化，发达经济体增速放缓，新兴经济体表现稳定。   \n",
       "3                                 全球贸易增长乏力。   \n",
       "5                            全球经济复苏预计将依旧疲软。   \n",
       "\n",
       "                                   gen_answer  \n",
       "0                                     中国银行研究院  \n",
       "1                                 2023年12月12日  \n",
       "2               2023年全球经济增长动力持续回落，经济增速连续两年下降。  \n",
       "3                   全球贸易增长乏力，主要经济体出口贸易同比增速下降。  \n",
       "5  预计2024年全球经济复苏将依旧疲软，主要经济体增长态势和货币政策走势将进一步分化。  "
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_dfs['glm4_plus'].head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7d79e974-089f-4c08-ba5e-804f6542e06a",
   "metadata": {},
   "source": [
    "# 评估"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "7aba460e-1260-49ed-b3c1-d963fc7d2fb4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-11T14:06:44.562776Z",
     "iopub.status.busy": "2024-10-11T14:06:44.562003Z",
     "iopub.status.idle": "2024-10-11T14:06:44.610316Z",
     "shell.execute_reply": "2024-10-11T14:06:44.609469Z",
     "shell.execute_reply.started": "2024-10-11T14:06:44.562706Z"
    }
   },
   "outputs": [],
   "source": [
    "import time\n",
    "from langchain_openai import ChatOpenAI\n",
    "\n",
    "judge_llm = ChatOpenAI(\n",
    "    api_key=os.environ['LLM_API_KEY'],\n",
    "    base_url=os.environ['LLM_BASE_URL'],\n",
    "    model_name='qwen2-72b-instruct',\n",
    "    temperature=0.0001\n",
    ")\n",
    "\n",
    "def evaluate(judge_llm, prediction_df):\n",
    "    \"\"\"\n",
    "    对预测结果进行打分\n",
    "    :param prediction_df: 预测结果，需要包含问题，参考答案，生成的答案，列名分别为question, ref_answer, gen_answer\n",
    "    :return 打分模型原始返回结果\n",
    "    \"\"\"\n",
    "    prompt_tmpl = \"\"\"\n",
    "你是一个经济学博士，现在我有一系列问题，有一个助手已经对这些问题进行了回答，你需要参照参考答案，评价这个助手的回答是否正确，仅回复“是”或“否”即可，不要带其他描述性内容或无关信息。\n",
    "问题：\n",
    "<question>\n",
    "{{question}}\n",
    "</question>\n",
    "\n",
    "参考答案：\n",
    "<ref_answer>\n",
    "{{ref_answer}}\n",
    "</ref_answer>\n",
    "\n",
    "助手回答：\n",
    "<gen_answer>\n",
    "{{gen_answer}}\n",
    "</gen_answer>\n",
    "请评价：\n",
    "    \"\"\"\n",
    "    results = []\n",
    "\n",
    "    for _, row in tqdm(prediction_df.iterrows(), total=len(prediction_df)):\n",
    "        question = row['question']\n",
    "        ref_answer = row['ref_answer']\n",
    "        gen_answer = row['gen_answer']\n",
    "\n",
    "        prompt = prompt_tmpl.replace('{{question}}', question).replace('{{ref_answer}}', str(ref_answer)).replace('{{gen_answer}}', gen_answer).strip()\n",
    "        result = judge_llm.invoke(prompt).content\n",
    "        results.append(result)\n",
    "\n",
    "        time.sleep(1)\n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "b628c548-13cc-4270-ab4d-714b13636ca7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-11T14:06:46.258787Z",
     "iopub.status.busy": "2024-10-11T14:06:46.258019Z",
     "iopub.status.idle": "2024-10-11T14:14:33.016638Z",
     "shell.execute_reply": "2024-10-11T14:14:33.014255Z",
     "shell.execute_reply.started": "2024-10-11T14:06:46.258717Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4f29c2f8d93842f6825020cd742776ae",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ollama_qwen2_7b unique raw score: ['是' '否']\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0f5660a02d39460ea97bc866d1d20887",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ollama_qwen2_7b_32k unique raw score: ['是' '否']\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d10374d66aa54d589863efa279c67b20",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "glm4_plus unique raw score: ['是' '否']\n"
     ]
    }
   ],
   "source": [
    "for key, pred_df in pred_dfs.items():\n",
    "    pred_df['raw_score'] = evaluate(judge_llm, pred_df)\n",
    "    pred_df['score'] = (pred_df['raw_score'] == '是').astype(int)\n",
    "    print(f\"{key} unique raw score: {pred_df['raw_score'].unique()}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "8b82fdf1-6012-4c1a-82e4-4f5f23438dcb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-11T14:16:26.827824Z",
     "iopub.status.busy": "2024-10-11T14:16:26.827062Z",
     "iopub.status.idle": "2024-10-11T14:16:26.836848Z",
     "shell.execute_reply": "2024-10-11T14:16:26.834549Z",
     "shell.execute_reply.started": "2024-10-11T14:16:26.827754Z"
    }
   },
   "outputs": [],
   "source": [
    "from IPython import display"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "88fc7227-9c21-48da-b179-5070406eb113",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-11T14:16:46.109050Z",
     "iopub.status.busy": "2024-10-11T14:16:46.108686Z",
     "iopub.status.idle": "2024-10-11T14:16:46.138261Z",
     "shell.execute_reply": "2024-10-11T14:16:46.137097Z",
     "shell.execute_reply.started": "2024-10-11T14:16:46.109018Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ollama_qwen2_7b accuracy: 0.06\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>qa_type</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>detailed</td>\n",
       "      <td>0.064516</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>large_context</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ollama_qwen2_7b_32k accuracy: 0.68\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>qa_type</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>detailed</td>\n",
       "      <td>0.666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>large_context</td>\n",
       "      <td>0.857143</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "glm4_plus accuracy: 0.86\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>qa_type</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>detailed</td>\n",
       "      <td>0.849462</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>large_context</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for key, pred_df in pred_dfs.items():\n",
    "    pred_df['score'] = (pred_df['raw_score'] == '是').astype(int)\n",
    "    print(f\"{key} accuracy: {pred_df['score'].mean()}\")\n",
    "\n",
    "    display.display_html(pred_df[['qa_type', 'score']].groupby('qa_type').mean().reset_index())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "7f82e38c-9f35-46b2-91d9-6575dc72b23b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-11T14:16:59.607253Z",
     "iopub.status.busy": "2024-10-11T14:16:59.606433Z",
     "iopub.status.idle": "2024-10-11T14:17:00.048612Z",
     "shell.execute_reply": "2024-10-11T14:17:00.046264Z",
     "shell.execute_reply.started": "2024-10-11T14:16:59.607181Z"
    }
   },
   "outputs": [],
   "source": [
    "for key, pred_df in pred_dfs.items():\n",
    "    pred_df.to_excel(os.path.join(expr_dir, f'{expr_version}_{key}_prediction.xlsx'), index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b9c3f8a4-00b9-4207-89f7-ed0e26131f8a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
